package crawl.topsy;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.util.HashMap;
import java.util.List;

import tools.FileWriterEnhance;
import tools.ObjectTools;

import com.maruti.otterapi.search.Post;

public class ReadResult {
	public void run(String path, String outputPath) {
		for (File dir : (new File(path).listFiles())) {
			if (dir.isDirectory()) {
				System.out.println(dir);
				StringBuilder sb1 = new StringBuilder();
				StringBuilder sb2 = new StringBuilder();
				HashMap<String, Boolean> outputed1 = new HashMap<>();
				
				for (File file : dir.listFiles()) {
					@SuppressWarnings("unchecked")
					List<Post> posts = (List<Post>) ObjectTools.readFromFile(file);
					for (Post post : posts) {
						String content = post.getContent().trim();
						if (!post.getTrackback_permalink().startsWith("http://twitter.com/")) {
							continue;
						}
						String twitterID = post.getTrackback_permalink().substring(post.getTrackback_permalink().lastIndexOf('/') + 1);
						try {
							Long.parseLong(twitterID);
						} catch (NumberFormatException e) {
							System.out.println("=> No TweetID: " + post.getTrackback_permalink() + "\t" + content);
							continue;
						}
//						// delete the duplicate tweets - method 1
//						if (outputed1.containsKey(post.getTrackback_permalink())) {
//							System.out.println("\tDuplicated: " + post.getTrackback_permalink() + "\t" + content);
//							continue;
//						} else {
//							outputed1.put(post.getTrackback_permalink(), true);
//							sb1.append(content.replaceAll("\\s+", " ").trim());
//							sb1.append('\n');
//							sb2.append(twitterID);
//							sb2.append('\n');
//						}
						// delete the duplicate tweets - method 2
						if (outputed1.containsKey(twitterID)) {
							System.out.println("\tDuplicated: " + twitterID + "\t" + content);
							continue;
						} else {
							outputed1.put(twitterID, true);
							sb1.append(content.replaceAll("\\s+", " ").trim());
							sb1.append('\n');
							sb2.append(twitterID);
							sb2.append('\n');
						}
					}
				}
				
				FileWriterEnhance fwe = new FileWriterEnhance(outputPath + "/" + dir.getName(), "UTF-8");
				fwe.WriteToFile(sb1.toString());
				FileWriterEnhance fwe1 = new FileWriterEnhance(outputPath + "_ID/"  + dir.getName(), "UTF-8");
				fwe1.WriteToFile(sb2.toString());
			} else {
				continue;
			}
		}
	}

	public static void main(String[] args) {
		String path = "data/objs/topsyCrawl/";
		String outputPath = "./data/_newData/plainText";
		
		try {
			System.setOut(new PrintStream(new File("./logs/ReadResult.out.log")));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}

		ReadResult rr = new ReadResult();
		rr.run(path, outputPath);
	}
}
